﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Microsoft.SharePoint;
using Microsoft.SharePoint.Publishing.WebControls;
using Microsoft.SharePoint.Utilities;
using Microsoft.SharePoint.WebPartPages;
using Microsoft.Office.Excel.WebUI;
using WoltersKluwer.SPChainGang.Core.Configuration;
using WoltersKluwer.SPChainGang.Core.Constants;
using WoltersKluwer.SPChainGang.Core.Data.Entity;
using WoltersKluwer.SPChainGang.Core.Data.Repository;
using WoltersKluwer.SPChainGang.Core.Util;
using WoltersKluwer.SPChainGang.Core.ExtensionMethods;
using System.IO;

namespace WoltersKluwer.SPChainGang.Core.ObjectScanners
{
    //TODO: this is a god-like class, need to split it into separate responsibilities
    public abstract class ScannerBase : IDisposable
    {
        readonly ConfigurationRepository configRepo;
        readonly CSVReportsRepository reportsRepo;
        readonly Hashtable checkedBrokenLinks = new Hashtable();
        private readonly ConfigurationSettings configuration;
        protected readonly JobMode OperationMode;
        protected static Guid JobId;
        protected SPSite targetSiteCollection;

        private ScannerBase(SPSite targetSiteCollection, Guid configurationListId, JobMode mode, Guid jobId)
        {
            configRepo = new ConfigurationRepository(targetSiteCollection.RootWeb, configurationListId);
            reportsRepo = new CSVReportsRepository(targetSiteCollection.RootWeb);
            OperationMode = mode;
            configuration = new ConfigurationSettings();
            JobId = jobId;
            this.targetSiteCollection = targetSiteCollection;
        }


        protected ScannerBase(string name, SPSite targetSiteCollection, Guid configurationListId, JobMode mode, Guid jobId)
            : this(targetSiteCollection, configurationListId, mode, jobId)
        {
            string filename = String.Format(GlobalConstants.ReportFolderNamePattern, name, OperationMode.ToFriendlyString(), DateTime.Now.ToString(GlobalConstants.TimeStampPattern));
            var escapedFilename = new StringBuilder(filename);
            foreach (var invalidChar in Path.GetInvalidFileNameChars())
            {
                escapedFilename.Replace(invalidChar, '-');
            }
            ReportTempFilename =
                reportsRepo.CreateNewReport(escapedFilename.ToString()
                    );
        }

        protected ScannerBase(SPSite targetSiteCollection, Guid configurationListId, JobMode mode, Guid jobId, string reportFolder, List<ConfigurationMapping> mappings)
            : this(targetSiteCollection, configurationListId, mode, jobId)
        {
            ReportTempFilename = reportFolder;
            this.mappings = mappings;
            mappingsLoaded = true;
        }

        public string ReportTempFilename { get; private set; }

        bool mappingsLoaded;
        List<ConfigurationMapping> mappings;
        protected List<ConfigurationMapping> Mappings
        {
            get
            {
                if (!mappingsLoaded)
                {
                    mappings = configRepo.GetConfigurationMapping();
                    mappingsLoaded = true;
                }
                return mappings;
            }
        }

        public abstract void Scan(SPWeb web);

        protected virtual string ProcessText(string text, Uri originPage, string additionalInfo)
        {
            return GlobalConstants.UrlMapper.Replace(text, match => LinkEvaluator(match, originPage, additionalInfo));
        }

        protected virtual string ProcessHtml(string htmlText, Uri originPage, string additionalInfo)
        {
            return GlobalConstants.UriMapper.Replace(htmlText, match => LinkEvaluator(match, originPage, additionalInfo));
        }

        protected virtual string ProcessLink(string oldLink, Uri originPage, string additionalInfo)
        {
            try
            {
                if (String.IsNullOrEmpty(oldLink))
                    return oldLink;

                var reportRecord = new ReportRecord
                                       {
                                           DetectedLink = oldLink,
                                           OriginPage = originPage.ToString(),
                                           AdditionalInfo = additionalInfo,
                                       };

                var oldUri = oldLink.GetAbsoluteUri(originPage);
                if (oldUri == null)
                {
                    Logger.Log(JobId, "Incorrect URL '{0}' at '{1}'", oldLink, originPage);
                    return oldLink;
                }

                if (oldUri.Scheme != Uri.UriSchemeHttp && oldUri.Scheme != Uri.UriSchemeHttps)
                    return oldLink; //this is probably an email or something

                if (!oldUri.Host.EqualsByContent(originPage.Host))
                    return oldLink; //it's a link to some external resource

                var mapping = GetMostQualifiedMapping(oldUri);
                if (mapping == null ||
                    (OperationMode == JobMode.CheckBrokenLinks && !IsLinkBroken(oldUri, reportRecord)))
                {
                    return oldLink;
                }
                reportRecord.AdditionalInfo = String.Format("Mapping {0}", mapping.Id);

                if (configuration.UpdateExternalLinksOnly && GetMostQualifiedMapping(originPage) != null)
                {
                    return oldLink;
                }

                var newLink = oldLink;
                if (OperationMode == JobMode.UpdateLinks && !mapping.NewUrlPattern.IsNullOrEmpty())
                {
                    var newLocalPath = new StringBuilder(oldUri.LocalPath);
                    newLocalPath.Remove(
                        0, LinkHelpers.MakeRooted(mapping.OldUrlPattern).Length).Insert(
                            0, LinkHelpers.MakeRooted(mapping.NewUrlPattern));

                    newLink = oldLink.Replace(oldUri.LocalPath, newLocalPath.ToString());

                    if (newLink.Equals(oldLink)) //old link may be (partially) encoded
                    {
                        newLink = oldLink.Replace(Uri.EscapeUriString(oldUri.LocalPath), Uri.EscapeUriString(newLocalPath.ToString()));
                    }

                    if (newLink.Equals(oldLink))
                    {
                        newLink = oldLink.Replace(SPEncode.UrlEncodeAsUrl(oldUri.LocalPath), SPEncode.UrlEncodeAsUrl(newLocalPath.ToString()));
                    }

                    if (newLink.Equals(oldLink))
                    {
                        newLink = Uri.UnescapeDataString(oldLink).Replace(oldUri.LocalPath, newLocalPath.ToString());
                        newLink = SPEncode.UrlEncodeAsUrl(newLink);
                    }

                    reportRecord.ReplacedWith = newLink;
                }
                if (OperationMode != JobMode.UpdateLinks || NeedUpdate(newLink, oldLink))
                    Report(reportRecord);

                return newLink;
            }
            catch (OutOfMemoryException)
            {
                throw;
            }
            catch (Exception e)
            {
                Logger.LogError(JobId, String.Format("Unable to process the link {0} found at {1}", oldLink, originPage), e);
                return oldLink;
            }
        }

        protected virtual void Report(ReportRecord reportRecord)
        {
            var record = reportsRepo.AddReportRecord(ReportTempFilename);
            record.LoadFrom(reportRecord);
            record.Update();
        }

        protected void ScanWebParts(SPFile file, Uri pageUri)
        {
            ScanWebParts(file, pageUri, false);
        }

        protected void ScanWebParts(SPFile file, Uri pageUri, bool versioningEnabled)
        {
#if SOS
            Logger.Log(JobId, "ScanWebParts() file={0}", file.Url);
#endif
            var modify = OperationMode == JobMode.UpdateLinks;
            SPLimitedWebPartManager wpm = null;
            try
            {
                wpm = file.GetLimitedWebPartManager(System.Web.UI.WebControls.WebParts.PersonalizationScope.Shared);
#if SOS
                Logger.Log(JobId, "ScanWebParts() GetLimitedWebPartManager");
                Logger.Log(JobId, "ScanWebParts() wpm.WebParts.Count={0}", wpm.WebParts.Count);

                SPLimitedWebPartManager wpm2 = null;
                try
                {
                    wpm2 = file.GetLimitedWebPartManager(System.Web.UI.WebControls.WebParts.PersonalizationScope.User);
                    Logger.Log(JobId, "ScanWebParts() wpm2.WebParts.Count={0}", wpm2.WebParts.Count);
                    foreach (WebPart wp2 in wpm2.WebParts)
                    {
                        using (wp2)
                            Logger.Log(JobId, "ScanWebParts() wp2={0} wp2.GetType()={1}", wp2.Title, wp2.GetType());
                    }
                }
                finally
                {
                    if (wpm2 != null)
                    {
                        if (wpm2.Web != null) wpm2.Web.Dispose();
                        wpm2.Dispose();
                    }                    
                }
#endif
                var webParts = wpm.WebParts;
                foreach (System.Web.UI.WebControls.WebParts.WebPart wp in webParts)
                {
                    try
                    {
                        string content;
#if SOS
                        Logger.Log(JobId, "ScanWebParts() wp={0} wp.GetType()={1}", wp.Title, wp.GetType());
#endif
                        if (wp is ContentEditorWebPart)
                        {
                            ContentEditorWebPart contentEditorWp = (ContentEditorWebPart)wp;

                            content = contentEditorWp.Content.InnerText;
#if SOS
                            Logger.Log(JobId, "ScanWebParts() ContentEditorWebPart content.Length={0}", content.Length);
                            if (pageUri.LocalPath.EndsWith("oldHome.aspx") ||
                                pageUri.LocalPath.EndsWith("CFS/Pages/default.aspx") ||
                                pageUri.LocalPath.EndsWith("LTRE/Pages/default.aspx") ||
                                pageUri.LocalPath.EndsWith("Home.aspx") ||
                                pageUri.LocalPath.EndsWith("aboutltre/mt/Pages/default.aspx") ||
                                pageUri.LocalPath.EndsWith("NewEmployees/Belgium/Pages/Default.aspx") ||
                                pageUri.LocalPath.EndsWith("NewEmployees/france/Pages/Default.aspx") ||
                                pageUri.LocalPath.EndsWith("NewEmployees/IT/Pages/Default.aspx") ||
                                pageUri.LocalPath.EndsWith("CFS/Corporate%20Legal%20Services/candg/default.aspx") ||
                                pageUri.LocalPath.EndsWith("CFS/Corporate%20Legal%20Services/candg/candgnews/default.aspx") ||
                                pageUri.LocalPath.EndsWith("/Financial%20Services/Pages/default.aspx") ||
                                pageUri.LocalPath.EndsWith("ConnectWKFS%20-%20Resources/Pages/default.aspx") ||
                                pageUri.LocalPath.EndsWith("Financial%20Services/Local%20Information/Pages/default.aspx") ||
                                pageUri.LocalPath.EndsWith("Brown%20Bag%20Presentations/Pages/Default.aspx")
                                )
                            {
                                Logger.Log(JobId, "ScanWebParts() ContentEditorWebPart content={0}",
                                           content);
                            }
#endif
                            var newContent = ProcessHtml(content, pageUri,
                                                         String.Format(
                                                             GlobalConstants.ReportAdditionalInfoWebPartPattern,
                                                             wp.Title, wp.GetType().Name));
                            if (modify && NeedUpdate(newContent, content))
                            {
                                XmlDocument xmlDoc = new XmlDocument();
                                XmlElement xmlElement = xmlDoc.CreateElement("HtmlContent");
                                xmlElement.InnerText = newContent;
                                contentEditorWp.Content = xmlElement;

                                SaveChangesInWebPart(file, contentEditorWp.ID, wpm,
                                                     webPartsCollection =>
                                                     ((ContentEditorWebPart)webPartsCollection[contentEditorWp.ID]).Content =
                                                     contentEditorWp.Content, versioningEnabled);
                            }
                        }
                        else if (wp is SummaryLinkWebPart)
                        {
                            var summaryLinkWp = (SummaryLinkWebPart)wp;

                            content = summaryLinkWp.SummaryLinkStore;
#if SOS
                            Logger.Log(JobId, "ScanWebParts() SummaryLinkWebPart content={0}", content);
#endif
                            var newContent = ProcessHtml(content, pageUri,
                                                         String.Format(
                                                             GlobalConstants.ReportAdditionalInfoWebPartPattern,
                                                             wp.Title, wp.GetType().Name));
                            if (modify && NeedUpdate(newContent, content))
                            {
                                summaryLinkWp.SummaryLinkStore = newContent;
                                SaveChangesInWebPart(file, summaryLinkWp.ID, wpm,
                                                     webPartsCollection =>
                                                     ((SummaryLinkWebPart)
                                                      webPartsCollection[summaryLinkWp.ID]).SummaryLinkStore =
                                                     summaryLinkWp.SummaryLinkStore, versioningEnabled);
                            }
                        }
                        else if (wp is PageViewerWebPart)
                        {
                            var pageViewWebPart = (PageViewerWebPart)wp;
                            string contentUrl = pageViewWebPart.ContentLink;
#if SOS
                            Logger.Log(JobId, "ScanWebParts() PageViewerWebPart contentUrl={0}", contentUrl);
#endif
                            try
                            {
                                var currentWeb = file.ParentFolder.ParentWeb;
                                var contentUri = contentUrl.GetAbsoluteUri(new Uri(currentWeb.Url));
#if SOS
                                Logger.Log(JobId, "ScanWebParts() PageViewerWebPart contentUri={0}", contentUri);
#endif
                                if (contentUri == null ||
                                    //changing only in the current SPSite
                                    !contentUri.ToString().StartsWith(currentWeb.Site.Url))
                                    continue;

                                SPFile pageViewWebPartFile;
                                using (var fileSite = new SPSite(contentUri.ToString()))
                                using (var fileWeb = fileSite.OpenWeb())
                                {
                                    pageViewWebPartFile = fileWeb.GetFile(contentUri.ToString());
                                    if (pageViewWebPartFile != null)
                                    {
                                        if (!IsContentFile(pageViewWebPartFile.Name)) continue;

                                        Uri fileLocation = LinkHelpers.MakeUri(fileWeb.Site.Url,
                                                                               pageViewWebPartFile.ServerRelativeUrl);

                                        var versioned = false;
                                        if (pageViewWebPartFile.InDocumentLibrary &&
                                            pageViewWebPartFile.ParentFolder != null)
                                        {
                                            var list = fileWeb.Lists[pageViewWebPartFile.ParentFolder.ParentListId];
                                            versioned = list != null && list.EnableVersioning;
                                        }
                                        ScanFileContents(pageViewWebPartFile, fileLocation, "Page Viewer Web Part",
                                                         versioned);
                                    }
                                }
                            }
                            catch (OutOfMemoryException)
                            {
                                throw;
                            }
                            catch (Exception e)
                            {
                                Logger.LogError(JobId, String.Format("PageViewerWebPart: unable to get content under '{0}'", contentUrl), e);
                            }
                        }
                        else if (wp is ContentByQueryWebPart)
                        {
                            var contentByQueryWebPart = (ContentByQueryWebPart)wp;
                            ProcessContentByQueryWebPart(pageUri, contentByQueryWebPart, wpm, file, versioningEnabled);
                        }
                        else if (wp is ExcelWebRenderer)
                        {
                            var excelWebRenderer = (ExcelWebRenderer)wp;
                            var newWorkbookLocation = ProcessLink(excelWebRenderer.WorkbookUri, pageUri,
                                                                  String.Format(
                                                                      GlobalConstants.ReportAdditionalInfoWebPartPattern,
                                                                      wp.Title, "Excel Web Access Web Part"));
                            if (modify && NeedUpdate(newWorkbookLocation, excelWebRenderer.WorkbookUri))
                            {
                                SaveChangesInWebPart(file, excelWebRenderer.ID, wpm,
                                                     webPartsCollection =>
                                                     ((ExcelWebRenderer) webParts[excelWebRenderer.ID]).WorkbookUri =
                                                     newWorkbookLocation, versioningEnabled);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Logger.LogError(JobId, String.Format("Unable to scan web part {0} on page {1}", wp.Title, pageUri), ex);
                    }
                    finally
                    {
                        wp.Dispose();
                    }
                }
            }
            catch (OutOfMemoryException)
            {
                throw;
            }
            catch (Exception e)
            {
                Logger.LogError(JobId, String.Format("Unable to scan web parts on page {0}", pageUri), e);
            }
            finally
            {
                if (wpm != null)
                {
                    if (wpm.Web != null) wpm.Web.Dispose();
                    wpm.Dispose();
                }
            }
        }

        private void ProcessContentByQueryWebPart(Uri pageUri, ContentByQueryWebPart contentByQueryWebPart, SPLimitedWebPartManager wpm, SPFile file, bool versioningEnabled)
        {
            if (contentByQueryWebPart.ListGuid.IsNullOrEmpty())
            {
                var newWebUrl = ProcessLink(contentByQueryWebPart.WebUrl, pageUri,
                                            String.Format(GlobalConstants.ReportAdditionalInfoWebPartPattern,
                                                          contentByQueryWebPart.Title,
                                                          "Content By Query Web Part - web url"));
                if (OperationMode == JobMode.UpdateLinks && NeedUpdate(newWebUrl, contentByQueryWebPart.WebUrl))
                {
                    SaveChangesInWebPart(file, contentByQueryWebPart.ID, wpm,
                                         webParts =>
                                         {
                                             ((ContentByQueryWebPart)
                                              webParts[contentByQueryWebPart.ID]).WebUrl = newWebUrl;
                                         }
                                         , versioningEnabled);
                }
                return;
            }

            //it references a list
            using (SPWeb web = targetSiteCollection.OpenWeb(contentByQueryWebPart.WebUrl))
            {
                if (!web.Exists) return;

                SPList list = GetReferencedList(contentByQueryWebPart, web);
                if (list == null) return;

                var newWebUrl = ProcessLink(contentByQueryWebPart.WebUrl, pageUri,
                                            String.Format(GlobalConstants.ReportAdditionalInfoWebPartPattern,
                                                          contentByQueryWebPart.Title,
                                                          "Content By Query Web Part - web url"));
                if (!NeedUpdate(newWebUrl, contentByQueryWebPart.WebUrl)) return;

                var oldLink = list.DefaultViewUrl;
                var newLink = ProcessLink(oldLink, pageUri,
                                          String.Format(GlobalConstants.ReportAdditionalInfoWebPartPattern,
                                                        contentByQueryWebPart.Title,
                                                        "Content By Query Web Part - list url"));
                if (OperationMode == JobMode.UpdateLinks && NeedUpdate(newLink, oldLink))
                {
                    using (SPSite site = new SPSite(newLink.GetAbsoluteUri(new Uri(web.Url)).ToString()))
                    using (SPWeb newWeb = site.OpenWeb())
                    {
                        var newList = newWeb.GetList(newLink);
                        SaveChangesInWebPart(file, contentByQueryWebPart.ID, wpm,
                                             webParts =>
                                                 {
                                                     ((ContentByQueryWebPart)
                                                      webParts[contentByQueryWebPart.ID]).ListGuid =
                                                         newList.ID.ToString();
                                                     ((ContentByQueryWebPart)
                                                      webParts[contentByQueryWebPart.ID]).WebUrl =
                                                         newWebUrl;
                                                 }
                                             , versioningEnabled);
                    }
                }
            }
        }

        private SPList GetReferencedList(ContentByQueryWebPart contentByQueryWebPart, SPWeb web)
        {
            SPList list = null;
            try
            {
                var listId = new Guid(contentByQueryWebPart.ListGuid);
                list = web.Lists[listId];
            }
            catch (SPException)
            {
              
            }
            return list;
        }

        protected static bool IsContentFile(string filename)
        {
            return IsAspxFile(filename) || IsHtmlFile(filename);
        }

        protected static bool IsAspxFile(string filename)
        {
            var fileExtension = Path.GetExtension(filename) ?? String.Empty;
            return fileExtension.StartsWith(".aspx");
        }

        protected static bool IsHtmlFile(string filename)
        {
            var fileExtension = Path.GetExtension(filename) ?? String.Empty;
            return fileExtension.StartsWith(".htm");
        }

        protected static bool IsPublishingPages(SPList list)
        {
            return list != null ? list.BaseTemplate.ToString() == "850" : false;
        }

        private static void SaveChangesInWebPart(SPFile file, string webPartId, SPLimitedWebPartManager wpm,
                                                 Action<SPLimitedWebPartCollection> updateWebPart,
                                                 bool versioningEnabled)
        {
#if SOS
            Logger.Log(JobId, "ScanWebParts() SaveChangesInWebPart() file.Item.Level={0}", file.Item.Level.ToString());
            Logger.Log(JobId, "ScanWebParts() SaveChangesInWebPart() file.CheckOutStatus={0}", file.CheckOutStatus.ToString());
#endif
            ProcessChangesOnPublishingPage(file, versioningEnabled, () =>
                {
                    if (versioningEnabled)
                    {
                        //need to reopen web parts manager again to reflect the checked out status
                        using (
                            var lwpw =
                                file.GetLimitedWebPartManager(
                                    System.Web.UI.WebControls.WebParts.PersonalizationScope.Shared))
                        {
                            using (lwpw.Web)
                            {
#if SOS
                        Logger.Log(JobId, "ScanWebParts() SaveChangesInWebPart() update web part");
#endif
                                var webParts = lwpw.WebParts;
                                updateWebPart(webParts);
                                lwpw.SaveChanges(webParts[webPartId]);
                            }
                        }
                    }
                    else
                    {
                        var webParts = wpm.WebParts;
                        updateWebPart(webParts);
                        wpm.SaveChanges(webParts[webPartId]);
                    }
                });
        }

        private static void SaveChangesInFile(SPFile file, bool versioningEnabled, byte[] content)
        {
            ProcessChangesOnPublishingPage(file, versioningEnabled, () =>
                {
                    file.SaveBinary(content);
                    file.Update();
                });
        }

        private static void ProcessChangesOnPublishingPage(SPFile file, bool versioningEnabled, Action processChanges)
        {
            if (versioningEnabled)
            {
                var list = file.GetListItem().ParentList;
                bool needToPublish = list.EnableMinorVersions && (file.Item.Level == SPFileLevel.Published);
                bool needToApprove = needToPublish && list.EnableModeration &&
                                     (file.Item.ModerationInformation.Status == SPModerationStatusType.Approved);
                
                EnsureReadyForCheckout(file);
                if (file.CheckOutType != SPFile.SPCheckOutType.None) return;
                file.CheckOut();
                processChanges();
                file.CheckIn("The links were updated by SPChainGang");

                string comment = "The file was re-published by SPChainGang";
                if (needToApprove) file.Approve(comment); else if (needToPublish) file.Publish(comment);
                file.Update();
            }
            else
            {
                processChanges();
            }

        }

        private static void EnsureReadyForCheckout(SPFile file)
        {
            if (file.CheckOutType == SPFile.SPCheckOutType.None) return;
            try
            {
                file.UndoCheckOut();
            }
            catch
            {
                file.CheckIn("Force check-in");
            }
            file.Update();
        }

        private string LinkEvaluator(Match match, Uri originPage, string additionalInfo)
        {
            var oldUrl = match.Groups["url"].Value;
            return oldUrl.IsNullOrEmpty() ? oldUrl : match.Value.Replace(oldUrl, ProcessLink(oldUrl, originPage, additionalInfo));
        }

        private ConfigurationMapping GetMostQualifiedMapping(Uri link)
        {
            if (link == null)
                return null;

            return
                Mappings.Where(
                    mapping =>
                    {
                        if (mapping.OldUrlPattern.IsNullOrEmpty() && mapping.NewUrlPattern.IsNullOrEmpty())
                            return false;

                        var path = link.LocalPath;
                        var oldUrlPattern = LinkHelpers.MakeRooted(mapping.OldUrlPattern);
                        return path.StartsWith(oldUrlPattern, StringComparison.InvariantCultureIgnoreCase) &&
                               (path.Length == oldUrlPattern.Length || path[oldUrlPattern.Length] == '/');
                    }).Max();
        }

        private bool IsLinkBroken(Uri urlToCheck, ReportRecord reportRecord)
        {
            //TODO move this stuff into a separate class

            var foundLink = urlToCheck.ToString();
            if (checkedBrokenLinks.ContainsKey(foundLink))
            {
                reportRecord.AdditionalInfo = "Duplicate";
                return (bool)checkedBrokenLinks[foundLink];
            }

            bool returnValue = false;
            if ((urlToCheck.Scheme == Uri.UriSchemeHttp) || (urlToCheck.Scheme == Uri.UriSchemeHttps))
            {
                ServicePointManager.ServerCertificateValidationCallback =
                    new System.Net.Security.RemoteCertificateValidationCallback(delegate { return true; });

                var request = (HttpWebRequest)WebRequest.Create(Uri.UnescapeDataString(urlToCheck.ToString()));
                if (!configuration.BrokenLinksCheckUsername.IsNullOrEmpty())
                {
                    NetworkCredential netCredential = new NetworkCredential(
                        configuration.BrokenLinksCheckUsername, configuration.BrokenLinksCheckPassword,
                        configuration.BrokenLinksCheckDomain);
                    request.CookieContainer = new CookieContainer();
                    request.Credentials = netCredential;
                    request.AllowAutoRedirect = false;
                }
                else
                {
                    Logger.Log(JobId, "Processing in Broken Links Check mode with no impersonation for the requests (no credentials in config)");
                    request.Credentials = CredentialCache.DefaultNetworkCredentials;
                }

                HttpWebResponse response = null;
                try
                {
                    response = (HttpWebResponse)request.GetResponse();
                    response.Close();
                }
                catch (WebException e)
                {
                    reportRecord.AdditionalInfo = String.Format("HTTP Status {0}", (int)((HttpWebResponse)e.Response).StatusCode);
                    returnValue = true;
                }
                finally
                {
                    if (response != null)
                        response.Close();
                }
            }

            checkedBrokenLinks.Add(foundLink, returnValue); //cache here
            return returnValue;
        }

        protected void ScanFileContents(SPFile file, Uri fileLocation)
        {
            ScanFileContents(file, fileLocation, null, false);
        }

        protected void ScanFileContents(SPFile file, Uri fileLocation, bool versioningEnabled)
        {
            ScanFileContents(file, fileLocation, null, versioningEnabled);
        }

        protected void ScanFileContents(SPFile file, Uri fileLocation, string info, bool versioningEnabled)
        {
#if SOS
            Logger.Log(JobId, "SitePagesScanner ScanFileContents() file={0}", file.Url);
#endif
            string fileContent = String.Empty;
#if SOS
            bool dumpContent = false;
#endif
            try
            {
                using (StreamReader reader = new StreamReader(file.OpenBinaryStream()))
                {
                    fileContent = reader.ReadToEnd();
#if SOS
                    Logger.Log(JobId, "SitePagesScanner ScanFileContents() fileContent.Length={0}", fileContent.Length);
                    /*
                    dumpContent = fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("") ||
                        fileLocation.LocalPath.EndsWith("");
                    */
                    if (dumpContent)
                    {
                        Logger.Log(JobId, "SitePagesScanner ScanFileContents() fileContent={0}", fileContent);
                    }
#endif
                }
            }
            catch (OutOfMemoryException)
            {
                throw;
            }
            catch (Exception e)
            {
                Logger.LogError(JobId, String.Format("Error reading file {0} ({1})", file.Title, fileLocation), e);
            }

            if (fileContent.IsNullOrEmpty()) return;

            try
            {
                var newContent = ProcessText(fileContent, fileLocation, info ?? "Site page");
#if SOS
                Logger.Log(JobId, "SitePagesScanner ScanFileContents() customizedInfo={0}", info);
                if (dumpContent)
                {
                    Logger.Log(JobId, "SitePagesScanner ScanFileContents() newContent={0}", newContent);
                }
#endif
                if (NeedUpdate(newContent, fileContent))
                {
                    var encoding = new UTF8Encoding();
                    var content = encoding.GetBytes(newContent);
                    SaveChangesInFile(file, versioningEnabled, content);
#if SOS
                    Logger.Log(JobId, "SitePagesScanner ScanFileContents() file.Update()");
#endif
                }
            }
            catch (OutOfMemoryException)
            {
                throw;
            }
            catch (Exception e)
            {
                Logger.LogError(JobId, String.Format("Error updating file {0} ({1})", file.Title, fileLocation), e);
            }
        }

        public void Dispose()
        {
            try
            {
                configRepo.Dispose();
            }
            finally
            {
                reportsRepo.Dispose();
            }
        }

        protected static bool NeedUpdate(string newContent, string oldContent)
        {
            return !newContent.IsNullOrEmpty() && !newContent.Equals(oldContent);
        }
    }
}
